#!/usr/bin/groovy
@Library('test-shared-library') _

properties(
        [
                pipelineTriggers([cron('H 16 * * *')]),
                buildDiscarder(logRotator(numToKeepStr: '10'))
        ]
)

static def awsRegion() {
    return "us-east-2"
}

def readPropertiesFile(file) {
    def properties = [:]
    readFile(file).split("\n").each { line ->
        if (!line.startsWith("#")) {
            def splits = line.split("=")
            properties[splits[0]] = splits[1]
        }
    }
    return properties
}

def runCommandWithK8sDebugInfo(GString script) {
    def result = sh(script: script, returnStatus: true)
    if (result > 0) {
        sh """
            kubectl describe service sparkling-water-app
            kubectl get pods --all-namespaces 
            kubectl describe pods
            kubectl logs sparkling-water-app
            kubectl get events
        """
    }
    return result
}

def runRCommandWithK8sDebugInfo(GString script) {
    def result = runCommandWithK8sDebugInfo(script)
    if (result > 0) {
        error("The script finished with the exit code ${result}.")
    }
}

static def getKubernetesSparkVersions(props) {
    def sparkVersions = props["supportedSparkVersions"].split(" ").toList()
    def boundaryVersion = props["kubernetesSupportSinceSpark"]
    def list = new ArrayList<String>()
    list.addAll(sparkVersions.subList(sparkVersions.indexOf(boundaryVersion), sparkVersions.size()))
    return list
}

static String getSparklingVersion(props, sparkMajorVersion) {
    return "${props['version'].replace("-SNAPSHOT", "")}-${sparkMajorVersion}"
}

String getSparkVersion(sparkMajorVersion) {
    def versionLine = readFile("gradle-spark${sparkMajorVersion}.properties").split("\n").find() { line -> line.startsWith('sparkVersion') }
    return versionLine.split("=")[1]
}

def buildAndPublishSparklingWaterImage(String type, String sparklingVersion, String repoUrl) {
    sh """
        H2O_HOME=${env.WORKSPACE}/h2o-3 ./bin/build-kubernetes-images.sh $type
        docker tag sparkling-water-$type:${sparklingVersion} $repoUrl/sw_kubernetes_repo/sparkling-water:$type-${sparklingVersion}
        docker push $repoUrl/sw_kubernetes_repo/sparkling-water:$type-${sparklingVersion}
        docker rmi sparkling-water-$type:${sparklingVersion}
        docker rmi $repoUrl/sw_kubernetes_repo/sparkling-water:$type-${sparklingVersion}
        """
}

String getH2OBranchMajorVersion() {
    def versionLine = readFile("h2o-3/gradle.properties").split("\n").find() { line -> line.startsWith('version') }
    return versionLine.split("=")[1]
}

String getH2OBranchMajorName() {
    return "bleeding_edge"
}

String getH2OBranchBuildVersion() {
    return "1-SNAPSHOT"
}

def getBuildAndPublishStage(props, sparkMajorVersion) {
    return {
        stage("Build & Publish Images For Spark $sparkMajorVersion") {
            node('docker') {
                try {
                    ws("${env.WORKSPACE}-spark-${sparkMajorVersion}") {
                        cleanWs()
                        checkout scm
                        def commons = load 'ci/commons.groovy'
                        commons.withSparklingWaterDockerImage {
                            sh "sudo apt-get update"
                            sh "sudo apt -y install docker.io"
                            sh "sudo service docker start"
                            sh "sudo chmod 666 /var/run/docker.sock"
                            def sparklingVersion = getSparklingVersion(props, sparkMajorVersion)
                            def sparkVersion = getSparkVersion(sparkMajorVersion)
                            unstash "shared"
                            sh """
                                sed -i 's/^h2oMajorName=.*\$/h2oMajorName=${getH2OBranchMajorName()}/' gradle.properties
                                sed -i 's/^h2oMajorVersion=.*\$/h2oMajorVersion=${getH2OBranchMajorVersion()}/' gradle.properties
                                sed -i 's/^h2oBuild=.*\$/h2oBuild=${getH2OBranchBuildVersion()}/' gradle.properties
                                """
                            sh "H2O_HOME=${env.WORKSPACE}/h2o-3 ./gradlew dist -Pspark=$sparkMajorVersion -Dmaven.repo.local=${env.WORKSPACE}/.m2 -PbuildAgainstH2OBranch=${props["testH2OBranch"]} -Ph2oMajorVersion=${getH2OBranchMajorVersion()} -Ph2oMajorName=${getH2OBranchMajorName()} -Ph2oBuild=${getH2OBranchBuildVersion()}"
                            def customEnv = [
                                    "SPARK_HOME=/home/jenkins/spark-${sparkVersion}-bin-hadoop2.7",
                            ]
                            unstash "properties"
                            def repoId = readPropertiesFile("properties")["docker_registry_id"]
                            def repoUrl = "${repoId}.dkr.ecr.${awsRegion()}.amazonaws.com"
                            docker.withRegistry("https://${repoUrl}", "ecr:${awsRegion()}:SW_FULL_AWS_CREDS") {
                                withEnv(customEnv) {
                                    dir("./dist/build/zip/sparkling-water-${sparklingVersion}") {
                                        buildAndPublishSparklingWaterImage("scala", sparklingVersion, repoUrl)
                                        buildAndPublishSparklingWaterImage("python", sparklingVersion, repoUrl)
                                        buildAndPublishSparklingWaterImage("r", sparklingVersion, repoUrl)
                                        sh """
                                            docker rmi spark-r:${sparkVersion}
                                            docker rmi spark-py:${sparkVersion}
                                            docker rmi spark:${sparkVersion}
                                           """
                                        buildAndPublishSparklingWaterImage("external-backend", sparklingVersion, repoUrl)

                                    }
                                }
                            }
                        }
                        cleanWs()
                    }
                } catch (Exception e) {
                    stopEKSCluster(commons)
                    throw e
                }
            }
        }
    }
}

def getBuildAndPublishStages(props) {
    def parallelStages = [:]
    getKubernetesSparkVersions(props).each { sparkMajorVersion ->
        parallelStages["Build & Publish Spark ${sparkMajorVersion}"] = getBuildAndPublishStage(props, sparkMajorVersion)
    }
    return parallelStages
}

def withTestEnv(commons, image, code) {
    def dockerCode = {
        commons.withAWSCredentials {
            unstash "properties"
            sh """
                apt-get update && apt-get install -y apt-transport-https gnupg2 curl unzip procps
                curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
                echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list
                apt-get update
                apt-get install -y kubectl
                kubectl version --client
                rm -rf awscliv2 && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
                unzip awscliv2.zip -d awscliv2
                ./awscliv2/aws/install -i /usr/local/aws-cli -b /usr/local/bin
                aws eks --region ${awsRegion()} update-kubeconfig --name ${readPropertiesFile("properties")["cluster_name"]}
                kubectl get svc
                kubectl delete clusterrolebinding default --ignore-not-found
                kubectl create clusterrolebinding default --clusterrole=edit --serviceaccount=default:default --namespace=default
                """
            code()
        }
    }
    def repoId = readPropertiesFile("properties")["docker_registry_id"]
    docker.withRegistry("https://${repoId}.dkr.ecr.${awsRegion()}.amazonaws.com", "ecr:${awsRegion()}:SW_FULL_AWS_CREDS") {
        commons.withDocker(image, dockerCode, "-u 0:0")
    }
}

def testScalaInternalBackendClusterMode(registryId, master, version) {
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo """
        \$SPARK_HOME/bin/spark-submit \
         --conf spark.kubernetes.container.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:scala-${version} \
         --conf spark.driver.host=sparkling-water-app \
         --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
         --conf spark.scheduler.minRegisteredResourcesRatio=1 \
         --master $master \
         --deploy-mode cluster \
         --name test \
         --class ai.h2o.sparkling.KubernetesTest \
         --conf spark.executor.instances=3 \
         local:///opt/sparkling-water/tests/kubernetesTest.jar
        """
    sh "sleep 60"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def testPythonInternalBackendClusterMode(registryId, master, version) {
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo """
        \$SPARK_HOME/bin/spark-submit \
         --conf spark.kubernetes.container.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:python-${version} \
         --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
         --conf spark.driver.host=sparkling-water-app \
         --conf spark.scheduler.minRegisteredResourcesRatio=1 \
         --master $master \
         --deploy-mode cluster \
         --name test \
         --conf spark.executor.instances=3 \
         local:///opt/sparkling-water/tests/initTest.py
        """
    sh "sleep 60"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def testRInternalBackend(registryId, master, version, sparkVersion) {
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runRCommandWithK8sDebugInfo """
        export KUBERNETES_MASTER=$master
        export REGISTRY_ID=$registryId
        export SW_VERSION=$version
        export SPARK_VERSION=$sparkVersion
        Rscript --default-packages=methods,utils /opt/sparkling-water/tests/initTest.R
        """
    sh 'sleep 60'
    sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
    sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
}

def installSparkHeadlessService() {
    sh """
kubectl delete service sparkling-water-app --ignore-not-found
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: sparkling-water-app
spec:
  clusterIP: "None"
  selector:
    spark-driver-selector: sparkling-water-app
EOF
        """
}

def testScalaInternalBackendClientMode(registryId, master, version) {
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:scala-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- \
        \$SPARK_HOME/bin/spark-submit \
         --conf spark.scheduler.minRegisteredResourcesRatio=1 \
         --conf spark.kubernetes.container.image=${image}  \
         --master $master \
         --name test \
         --class ai.h2o.sparkling.KubernetesTest \
         --conf spark.driver.host=sparkling-water-app \
         --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
         --deploy-mode client \
         --conf spark.executor.instances=3 \
         local:///opt/sparkling-water/tests/kubernetesTest.jar
        """
    sh "sleep 60"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def testPythonInternalBackendClientMode(registryId, master, version) {
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:python-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- \
        \$SPARK_HOME/bin/spark-submit \
        --conf spark.scheduler.minRegisteredResourcesRatio=1 \
        --conf spark.kubernetes.container.image=${image}  \
        --master $master \
        --name test \
        --conf spark.driver.host=sparkling-water-app \
        --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
        --deploy-mode client \
        --conf spark.executor.instances=3 \
        local:///opt/sparkling-water/tests/initTest.py
        """
    sh "sleep 60"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def installH2OHeadlessService() {
    sh "kubectl delete service h2o-service --ignore-not-found"
    sh """
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: h2o-service
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: h2o-k8s
  ports:
  - protocol: TCP
    port: 54321
EOF
    """
}

def startExternalH2OBackend(registryId, version) {
    installH2OHeadlessService()
    sh "kubectl delete statefulsets h2o-stateful-set --ignore-not-found"
    sh """
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: h2o-stateful-set
  namespace: default
spec:
  serviceName: h2o-service
  replicas: 2
  selector:
    matchLabels:
      app: h2o-k8s
  template:
    metadata:
      labels:
        app: h2o-k8s
    spec:
      terminationGracePeriodSeconds: 10
      containers:
        - name: h2o-k8s
          image: '${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:external-backend-${version}'
          resources:
            requests:
              memory: "2Gi"
          ports:
            - containerPort: 54321
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /kubernetes/isLeaderNode
              port: 8081
            initialDelaySeconds: 5
            periodSeconds: 5
            failureThreshold: 1
          env:
          - name: H2O_KUBERNETES_SERVICE_DNS
            value: h2o-service.default.svc.cluster.local
          - name: H2O_NODE_LOOKUP_TIMEOUT
            value: '180'
          - name: H2O_NODE_EXPECTED_COUNT
            value: '2'
          - name: H2O_KUBERNETES_API_PORT
            value: '8081'
EOF
    """
    sh 'sleep 60'
}

def stopExternalH2OBackend() {
    sh "kubectl delete statefulsets h2o-stateful-set --ignore-not-found"
    sh "kubectl delete service h2o-service --ignore-not-found"
}

def testRExternalBackendManual(registryId, master, version, sparkVersion) {
    startExternalH2OBackend(registryId, version)
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runRCommandWithK8sDebugInfo """
        export KUBERNETES_MASTER=$master
        export REGISTRY_ID=$registryId
        export SW_VERSION=$version
        export SPARK_VERSION=$sparkVersion
        export EXTRA_OPTIONS="spark.ext.h2o.backend.cluster.mode=external spark.ext.h2o.external.start.mode=manual spark.ext.h2o.external.memory=2G spark.ext.h2o.cloud.representative=h2o-service.default.svc.cluster.local:54321 spark.ext.h2o.cloud.name=root"
        Rscript --default-packages=methods,utils /opt/sparkling-water/tests/initTest.R
        """
    sh 'sleep 60'
    sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
    sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

def testRExternalBackendAuto(registryId, master, version, sparkVersion) {
    sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runRCommandWithK8sDebugInfo """
        export KUBERNETES_MASTER=$master
        export REGISTRY_ID=$registryId
        export SW_VERSION=$version
        export SPARK_VERSION=$sparkVersion
        export EXTRA_OPTIONS="spark.ext.h2o.backend.cluster.mode=external spark.ext.h2o.external.k8s.namespace=h2o spark.ext.h2o.external.start.mode=auto spark.ext.h2o.external.cluster.size=2 spark.ext.h2o.external.k8s.docker.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:external-backend-${version} spark.ext.h2o.external.auto.start.backend=kubernetes spark.ext.h2o.external.memory=2G"
        Rscript --default-packages=methods,utils /opt/sparkling-water/tests/initTest.R
        """
    sh 'sleep 60'
    sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
    sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

static String externalBackendManualSharedSubmitCmd(registryId, master, version, mode, language) {
    return """\$SPARK_HOME/bin/spark-submit \
             --conf spark.kubernetes.container.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:${language}-${version} \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode $mode \
             --name test \
             --conf spark.ext.h2o.backend.cluster.mode=external \
             --conf spark.ext.h2o.external.start.mode=manual \
             --conf spark.ext.h2o.external.memory=2G \
             --conf spark.ext.h2o.cloud.representative=h2o-service.default.svc.cluster.local:54321 \
             --conf spark.ext.h2o.cloud.name=root \
             --conf spark.executor.instances=2 \
            """
}

static String externalBackendAutoSharedSubmitCmd(registryId, master, version, mode, language) {
    return """\$SPARK_HOME/bin/spark-submit \
             --conf spark.kubernetes.container.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:${language}-${version} \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode $mode \
             --name test \
             --conf spark.ext.h2o.backend.cluster.mode=external \
             --conf spark.ext.h2o.external.start.mode=auto \
             --conf spark.ext.h2o.external.auto.start.backend=kubernetes \
             --conf spark.ext.h2o.external.memory=2G \
             --conf spark.ext.h2o.external.k8s.namespace=h2o \
             --conf spark.ext.h2o.external.k8s.docker.image=${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:external-backend-${version} \
             --conf spark.executor.instances=2 \
             --conf spark.ext.h2o.external.cluster.size=2 \
            """
}

static String scalaExternalBackendSubmitCmd(registryId, master, version, mode, startMode) {
    def prefix
    if (startMode == "manual") {
        prefix = externalBackendManualSharedSubmitCmd(registryId, master, version, mode, "scala")
    } else {
        prefix = externalBackendAutoSharedSubmitCmd(registryId, master, version, mode, "scala")
    }
    return """${prefix} \
              --class ai.h2o.sparkling.KubernetesTest \
              local:///opt/sparkling-water/tests/kubernetesTest.jar
           """
}

def testScalaExternalBackendManualClusterMode(registryId, master, version) {
    startExternalH2OBackend(registryId, version)
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo "${scalaExternalBackendSubmitCmd(registryId, master, version, "cluster", "manual")}"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

def testScalaExternalBackendManualClientMode(registryId, master, version) {
    startExternalH2OBackend(registryId, version)
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:scala-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- ${scalaExternalBackendSubmitCmd(registryId, master, version, "client", "manual")}
        """
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

def testScalaExternalBackendAutoClusterMode(registryId, master, version) {
    sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo "${scalaExternalBackendSubmitCmd(registryId, master, version, "cluster", "auto")}"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def testScalaExternalBackendAutoClientMode(registryId, master, version) {
    sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:scala-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- ${scalaExternalBackendSubmitCmd(registryId, master, version, "client", "auto")}
        """
    sh "sleep 420"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

static String pythonExternalBackendSubmitCmd(registryId, master, version, mode, startMode) {
    def prefix
    if (startMode == "manual") {
        prefix = externalBackendManualSharedSubmitCmd(registryId, master, version, mode, "python")
    } else {
        prefix = externalBackendAutoSharedSubmitCmd(registryId, master, version, mode, "python")
    }
    return """$prefix \
              local:///opt/sparkling-water/tests/initTest.py
            """
}


def testPythonExternalBackendManualClusterMode(registryId, master, version) {
    startExternalH2OBackend(registryId, version)
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo "${pythonExternalBackendSubmitCmd(registryId, master, version, "cluster", "manual")}"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

def testPythonExternalBackendManualClientMode(registryId, master, version) {
    startExternalH2OBackend(registryId, version)
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:python-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- ${pythonExternalBackendSubmitCmd(registryId, master, version, "client", "manual")}
        """
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    stopExternalH2OBackend()
}

def testPythonExternalBackendAutoClusterMode(registryId, master, version) {
    sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    runCommandWithK8sDebugInfo "${pythonExternalBackendSubmitCmd(registryId, master, version, "cluster", "auto")}"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def testPythonExternalBackendAutoClientMode(registryId, master, version) {
    sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
    sh "kubectl delete pod sparkling-water-app --ignore-not-found"
    installSparkHeadlessService()
    def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:python-${version}"
    runCommandWithK8sDebugInfo """
        kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
        --image=${image} -- ${pythonExternalBackendSubmitCmd(registryId, master, version, "client", "auto")}
        """
    sh "sleep 420"
    sh "kubectl logs sparkling-water-app"
    sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
}

def stopEKSCluster(commons) {
    commons.withAWSCredentials {
        dir("kubernetes/src/terraform/aws") {
            sh "rm -f terraform.tfstate"
            unstash "tf_state"
            def code = {
                commons.terraformDestroy("-var aws_region=${awsRegion()}")
            }
            retryWithDelay(3, 600, {
                commons.withDocker("hashicorp/terraform:0.12.25", code, "--entrypoint='' --network host")
            })
        }
    }
}

node("docker") {
    cleanWs()
    checkout scm
    def commons = load 'ci/commons.groovy'
    def props = readPropertiesFile("gradle.properties")
    def sparklingVersion = props['version'].replace("-SNAPSHOT", "")

    stage("Build H2O") {
        commons.withSparklingWaterDockerImage {
            retryWithDelay(3, 60, {
                sh "git clone https://github.com/h2oai/h2o-3.git"
            })
            retryWithDelay(5, 1, {
                sh """
                    cd h2o-3
                    git checkout ${props["testH2OBranch"]}
                    . /envs/h2o_env_python2.7/bin/activate
                    ./gradlew build --parallel -x check -Duser.name=ec2-user
                    ./gradlew publishToMavenLocal --parallel -Dmaven.repo.local=${env.WORKSPACE}/.m2 -Duser.name=ec2-user -Dhttp.socketTimeout=600000 -Dhttp.connectionTimeout=600000
                    ./gradlew :h2o-r:buildPKG -Duser.name=ec2-user
                    cd ..
                    """
            })
        }
        stash name: "shared", excludes: "h2o-3/h2o-py/h2o/**/*.pyc, h2o-3/h2o-py/h2o/**/h2o.jar", includes: "h2o-3/build/h2o.jar, h2o-3/h2o-dist/buildinfo.json, h2o-3/gradle.properties, .m2/**, h2o-3/h2o-py/h2o/**, h2o-3/h2o-r/h2o_*.99999.tar.gz"
    }

    stage("Start EKS & ECR") {
        commons.withAWSCredentials {
            dir("kubernetes/src/terraform/aws") {
                def code = {
                    commons.terraformApply("-var aws_region=${awsRegion()}")
                    def values = commons.extractTerraformOutputs(["docker_registry_id", "cluster_name", "cluster_endpoint"])
                    def valuesAsString = values.collect { "${it.key}=${it.value}" }.join("\n") + "\n" + "version=$sparklingVersion" + "\n"
                    writeFile file: "properties", text: valuesAsString
                    stash name: "properties", includes: "properties"
                    arch "terraform.tfstate"
                    stash name: "tf_state", includes: "terraform.tfstate"
                }
                commons.withDocker("hashicorp/terraform:0.12.25", code, "--entrypoint='' --network host")
            }
        }
    }

    parallel(getBuildAndPublishStages(props))

    getKubernetesSparkVersions(props).each { sparkMajorVersion ->
        stage("Test Scala, Spark ${sparkMajorVersion}") {
            try {
                unstash "properties"
                def registryId = readPropertiesFile("properties")["docker_registry_id"]
                def version = readPropertiesFile("properties")["version"] + "-${sparkMajorVersion}"
                def master = "k8s://" + readPropertiesFile("properties")["cluster_endpoint"]
                def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:scala-${version}"
                withTestEnv(commons, image) {
                    testScalaInternalBackendClusterMode(registryId, master, version)
                    testScalaInternalBackendClientMode(registryId, master, version)
                    testScalaExternalBackendManualClusterMode(registryId, master, version)
                    testScalaExternalBackendManualClientMode(registryId, master, version)
                    testScalaExternalBackendAutoClusterMode(registryId, master, version)
                    testScalaExternalBackendAutoClientMode(registryId, master, version)
                }
            } catch (Exception e) {
                stopEKSCluster(commons)
                throw e
            }
        }

        stage("Test Python, Spark ${sparkMajorVersion}") {
            try {
                unstash "properties"
                def registryId = readPropertiesFile("properties")["docker_registry_id"]
                def version = readPropertiesFile("properties")["version"] + "-${sparkMajorVersion}"
                def master = "k8s://" + readPropertiesFile("properties")["cluster_endpoint"]
                def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:python-${version}"
                withTestEnv(commons, image) {
                    testPythonInternalBackendClusterMode(registryId, master, version)
                    testPythonInternalBackendClientMode(registryId, master, version)
                    testPythonExternalBackendManualClusterMode(registryId, master, version)
                    testPythonExternalBackendManualClientMode(registryId, master, version)
                    testPythonExternalBackendAutoClusterMode(registryId, master, version)
                    testPythonExternalBackendAutoClientMode(registryId, master, version)
                }
            } catch (Exception e) {
                stopEKSCluster(commons)
                throw e
            }
        }

        stage("Test R, Spark ${sparkMajorVersion}") {
            try {
                unstash "properties"
                def registryId = readPropertiesFile("properties")["docker_registry_id"]
                def version = readPropertiesFile("properties")["version"] + "-${sparkMajorVersion}"
                def master = "k8s://" + readPropertiesFile("properties")["cluster_endpoint"]
                def image = "${registryId}.dkr.ecr.${awsRegion()}.amazonaws.com/sw_kubernetes_repo/sparkling-water:r-${version}"
                def sparkVersion = getSparkVersion(sparkMajorVersion)
                withTestEnv(commons, image) {
                    // R sets deployment mode automatically
                    testRInternalBackend(registryId, master, version, sparkVersion)
                    testRExternalBackendManual(registryId, master, version, sparkVersion)
                    testRExternalBackendAuto(registryId, master, version, sparkVersion)
                }
            } catch (Exception e) {
                stopEKSCluster(commons)
                throw e
            }
        }
    }

    stage("Stop EKS & ECR") {
        stopEKSCluster(commons)
    }
}
